{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-02T12:25:39.829297Z",
     "start_time": "2025-09-02T12:25:39.601441Z"
    }
   },
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pendulum-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Gym has been unmaintained since 2022 and does not support NumPy 2.0 amongst other critical functionality.\n",
      "Please upgrade to Gymnasium, the maintained drop-in replacement of Gym, or contact the authors of your software and request that they upgrade.\n",
      "Users of this version of Gym should be able to simply replace 'import gym' with 'import gymnasium as gym' in the vast majority of cases.\n",
      "See the migration guide at https://gymnasium.farama.org/introduction/migration_guide/ for additional information.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([-0.36813426,  0.9297727 , -0.49744996], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-02T12:25:40.488652Z",
     "start_time": "2025-09-02T12:25:39.930322Z"
    }
   },
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjUsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvWftoOwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAIUpJREFUeJzt3Q1wVPW9//Hvbp6fQwIk5JIIXlBAHsoz0fZ6b4lEpVaU3rGOQ1PLyJQCA+JwK63i6L9jGOyt1VaxdzpV5k6FXhzBgqBmAgTRCCGAQoT4UJ4EkhAgmwfI8/nP72d3m6VRE/L03c37NXM82XPO7p79meyH3+98zzkux3EcAQBAIXdf7wAAAF+FkAIAqEVIAQDUIqQAAGoRUgAAtQgpAIBahBQAQC1CCgCgFiEFAFCLkAIAqNVnIfXCCy/IsGHDJDIyUqZPny779u3rq10BACjVJyH1l7/8RZYvXy5PPPGEHDhwQCZMmCDZ2dlSUVHRF7sDAFDK1RcXmDU9p6lTp8rvf/97+7i1tVXS09NlyZIl8uijj/b27gAAlArt7TdsbGyU4uJiWblypW+Z2+2WrKwsKSwsbPc5DQ0NdvIyoXbx4kVJTk4Wl8vVK/sNAOg+pn9UU1MjaWlpNgPUhFRlZaW0tLRISkqK33Lz+NixY+0+Jzc3V5588sle2kMAQG85ffq0DB06VE9IXQvT6zLHsLw8Ho9kZGTYDxcfH9+n+wYA6Lzq6mp7mCcuLu5rt+v1kBo4cKCEhIRIeXm533LzODU1td3nRERE2OlqJqAIKQAIXN90yKbXq/vCw8Nl8uTJkp+f73eMyTzOzMzs7d0BACjWJ8N9ZuguJydHpkyZItOmTZPf/va3UldXJw8++GBf7A4AQKk+Can77rtPzp8/L6tWrZKysjL51re+JW+99dY/FVMAAPq3PjlPqjsOuCUkJNgCCo5JAUDg6ej3ONfuAwCoRUgBANQipAAAahFSAAC1CCkAgFqEFABALUIKAKAWIQUAUIuQAgCoRUgBANQipAAAahFSAAC1CCkAgFqEFABALUIKAKAWIQUAUIuQAgCoRUgBANQipAAAahFSAAC1CCkAgFqEFABALUIKAKAWIQUAUIuQAgCoRUgBANQipAAAahFSAAC1CCkAgFqEFABALUIKAKAWIQUAUIuQAgCoRUgBANQipAAAahFSAAC1CCkAgFqEFABALUIKAKAWIQUAUIuQAgCoRUgBANQipAAAahFSAAC1CCkAgFqEFABALUIKAKAWIQUAUIuQAgCoRUgBANQipAAAahFSAAC1CCkAgFqEFABALUIKAKAWIQUAUIuQAgCoRUgBANQipAAAahFSAAC1CCkAgFqEFABALUIKABA8IbV792656667JC0tTVwul2zevNlvveM4smrVKhkyZIhERUVJVlaWfPrpp37bXLx4UR544AGJj4+XxMREmT9/vtTW1nb90wAA+ndI1dXVyYQJE+SFF15od/2aNWvk+eefl5deekn27t0rMTExkp2dLfX19b5tTECVlJRIXl6ebN261QbfggULuvZJAADBx+kC8/RNmzb5Hre2tjqpqanOM88841tWVVXlREREOOvXr7ePP/74Y/u8oqIi3zbbt293XC6Xc+bMmQ69r8fjsa9h5gCAwNPR7/FuPSZ1/PhxKSsrs0N8XgkJCTJ9+nQpLCy0j83cDPFNmTLFt43Z3u12255XexoaGqS6utpvAgAEv24NKRNQRkpKit9y89i7zswHDx7stz40NFSSkpJ821wtNzfXhp13Sk9P787dBgAoFRDVfStXrhSPx+ObTp8+3de7BAAItJBKTU218/Lycr/l5rF3nZlXVFT4rW9ubrYVf95trhYREWErAdtOAIDg160hNXz4cBs0+fn5vmXm+JE51pSZmWkfm3lVVZUUFxf7ttmxY4e0trbaY1cAAHiFSieZ85k+++wzv2KJQ4cO2WNKGRkZsmzZMvnVr34lI0eOtKH1+OOP23Oq5syZY7cfPXq03H777fLQQw/ZMvWmpiZZvHix/PCHP7TbAQDg09mywZ07d9qywaunnJwcXxn6448/7qSkpNjS85kzZzqlpaV+r3HhwgXn/vvvd2JjY534+HjnwQcfdGpqarq9dBEAoFNHv8dd5j8SYMwQoqnyM0UUHJ8CgMDT0e/xgKjuAwD0T4QUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAADBEVK5ubkydepUiYuLk8GDB8ucOXOktLTUb5v6+npZtGiRJCcnS2xsrMydO1fKy8v9tjl16pTMnj1boqOj7eusWLFCmpubu+cTAQD6Z0gVFBTYAPrggw8kLy9PmpqaZNasWVJXV+fb5uGHH5YtW7bIxo0b7fZnz56Ve++917e+paXFBlRjY6O8//77sm7dOnnllVdk1apV3fvJAACBz+mCiooKx7xEQUGBfVxVVeWEhYU5Gzdu9G1z9OhRu01hYaF9vG3bNsftdjtlZWW+bdauXevEx8c7DQ0NHXpfj8djX9PMAQCBp6Pf4106JuXxeOw8KSnJzouLi23vKisry7fNqFGjJCMjQwoLC+1jMx83bpykpKT4tsnOzpbq6mopKSlp930aGhrs+rYTACD4XXNItba2yrJly+SWW26RsWPH2mVlZWUSHh4uiYmJftuaQDLrvNu0DSjveu+6rzoWlpCQ4JvS09OvdbcBAP0hpMyxqSNHjsiGDRukp61cudL22rzT6dOne/w9AQB9L/RanrR48WLZunWr7N69W4YOHepbnpqaagsiqqqq/HpTprrPrPNus2/fPr/X81b/ebe5WkREhJ0AAP1Lp3pSjuPYgNq0aZPs2LFDhg8f7rd+8uTJEhYWJvn5+b5lpkTdlJxnZmbax2Z++PBhqaio8G1jKgXj4+NlzJgxXf9EAID+2ZMyQ3yvvvqqvPHGG/ZcKe8xJHOcKCoqys7nz58vy5cvt8UUJniWLFlig2nGjBl2W1OybsJo3rx5smbNGvsajz32mH1teksAgLZcpsRPOsjlcrW7/OWXX5Yf//jHvpN5H3nkEVm/fr2tyjOVey+++KLfUN7Jkydl4cKFsmvXLomJiZGcnBxZvXq1hIZ2LDNNdZ8JRHN8ygQhACCwdPR7vFMhpQUhBQCBraPf41y7DwCgFiEFAFCLkAIAqEVIAQDUIqQAAGoRUgAAtQgpAIBahBQAQC1CCgCgFiEFAFCLkAIAqEVIAQDUIqQAAGoRUgAAtQgpAIBahBQAQC1CCgCgFiEFAFCLkAIAqBXa1zsA4B8cx/nKdS6Xq1f3BdCAkAIUBFPr5cvSXFsrNSUl4ikqkvrTp6XlyhUJjY+XmBEjZMC3vy3R//qvEhITQ1ihXyGkgD7UWFkpnv37pebIERtOrVeu+K1vOn9ernz+uVzYuVMSJk2SwXPmSOzo0QQV+g1CCuhF3uG8+i++kIsFBVJ75IjUHjsm0tr69c9rbJSqDz6Q+nPnJGPBAokdO5agQr9ASAG9oLW5WZqrq+XK8eNS9tprdt7S0CDS0tKp16k/eVJO/c//SMbChfSo0C8QUkAPclpa7FCenT76SOqOHv3G55ypq5ODFy9KTVOTDIqMlMxBgyQmLMwvqM6tXy/X/9d/SWhcXA9/AqBvEVJAN3NaW6W1vl5qDh+2x5LqPvlEmiorv/l5jiPHa2vliYMH5URtrdS3tEh8WJiMHTBAfj11qoS5/3HGSM2HH8rlzz+XuAkT6E0hqLmcr6t5Vaq6uloSEhLE4/FIfHx8X+8OYAOm5fJlG0ZV+/dL2f/9nzjNzeI0NXX4NT6vqZEF770nnnaeM23gQPl/EydKcmSkb1lEWprctHYtIYWA1NHvcXpSQBc1XboklwoLpa601PZwmi5evKbX+a0pP/+KUNtXWSl5Z8/KD6+/3resMwEIBCpCCriGXpPpJZny8Qs7dnx5XtMXX9gKPADdi5ACOsgG06VLUn/ihJRv2WLLx01hhATeiDkQMAgpoAPMuUyX3n1XLp84IbUff9zp0vGOmJ2eLvsrK6WpndAbFhsr45OSuv09Ae0IKaAdpodkLktUW1Iilfn5cvmzzzpUodcV2Wlpdv6rDz+UxpYWMaf3hrhckhgeLv89dapcFxvrt33K3Lk9uj+ABoQU0Iap0DPHl8x5TWWvvy4tNTW9NpxnqvRMUA2NjpatX3whF+rrbQ/qvuHDJTkiwm/biCFDZEBmJpV9CHqEFCAizXV1cmn3bnuB17pjx6SxoqJP9sOEjjkvykxfJXTAAEmbN89efBYIdoQU+u+VxxsapPH8eXtNPFOl13Thgj0JV7OQ2FgZct99kjhtmrhCQvp6d4AeR0ih3x1rMsF05cQJezWIqsJCCQQmkCLT0yXlnnsk6d//nWE+9BuEFPqNK6dOyaU9e+zlikwhhOlJBYqo4cPluiVLJCojg4BCv0JIIajPa2quqZG6zz6TyrfestfQa6mrs8sDQUh0tISnpkrSd74jyTNnSlhiYl/vEtDrCCkEnZb6ejucZyr0Kt95RxrLyiRgmGG9IUPsXXgHfOc7Ejd2rLijoug9od8ipBDwvNdINsebLprjTHv32jLyhrNnJZBEjxxpjzeZ+0SZYT13eHhf7xLQ5wgpBPx5TQ3nzonn4EG5kJcnjRcuBMw19FxhYRKakGBvtzEwK0uirrtOQkyviao9wIeQQkD2nEyFnrnqePWBA7Z8PJCun2fKyGNuuMEO5SVMny6R//Iv4mpzrygA/0BIIXCG9BxHGsrLpXzzZrnyt79J/enTticVEFwuO3yXfNttkjBpkq3WC09O7uu9AtQjpKD/vKbKSrly8qRcyM+X6oMHvywdD5CekxnOM+c3DbjlFhlw8822F+UKDaUQAuggQgoqmSC6fPKk1Bw6ZM9tMtV6AcPlkqhhw+yQXsLUqRI3fryEtLmjLoCOI6SgajjPaW0Vz759UrFli73DbYMpHw+EXpPpGbndEn399TLo9tslZtQoiRg8WNxXXRgWQOcQUuhz5riSGc4zV4Iwlyoy5zXZmwkGADN0F5GWJrE33eSr0DPLTGgxpAd0HSGFPtN06ZJUHzokNR99JJfee0/9xV3bComLs8FkzmlKnDFDIlJSqNADegAhhV4d0jOXJDLhZCr0zO3XTSm5uVRRILAFDyEhMujOO20hRHhKir1dBj0moOcQUuiVcDIn3JqLunr277eFEHY4LxCONf39/k3mWFPSv/2bxE+aJKFxcQznAb2EkEKPMb2my8ePi6e4WKqLi+3Jt4Ek6vrrJX7iRIkbM8ZW6LnCwwkmoJcRUuje4by/36/JnNPkKSqS5upqaa6qCoxCCLfbVuNFpqVJ6n/+p72WnrnyuDssrK/3DOi3CCl0G1MyXpmXJ+fffFOaPR4JFO7oaHtB19gxY+yQnjnHyZaU//127gD6DiGFbtFQUSFn16+XS7t2BUavyYRTVJQkTp9uh/RMQJkKPQC6EFLosubaWjnzv/8rVe+9pz6g3JGREj54sL1Ekbkthh3Oi4ykfBxQipBCl53ftk0uFRSIWm63RKSm2hNtzY0ETc/J3PXWYDgP0I2QQpeYc54q335btDLHl5JuvVVix4798n5NXEMPCCiEFLrEnJRrbjSohTnZ1ns1iEGzZtnbsJtek71UEYCAw18uuqTlyhWR1ta+3g0JiYmx5zWZc5oSb7nFVutxnAkIfIQUAprpOSVnZUn8lCkSbW4kOGgQx5mAIEJIIeCYXpO58rgpghiQmSlhAwaIKyyMcAKCECGFgBGZkSExI0fa6+clTJkiIVFRfb1LAHoYIYUuMT0Zc5uNlpqannkDt9veft3cSNCecDtkCBV6QD/SqSPLa9eulfHjx0t8fLydMjMzZfv27b719fX1smjRIklOTpbY2FiZO3eulJeX+73GqVOnZPbs2RIdHS2DBw+WFStWSHNzc/d9IvSquAkT7BXCu/28prQ0Sfrud+WGp5+W0b/+tQy64w5bTk5AAf1Lp3pSQ4cOldWrV8vIkSPtxUTXrVsnd999txw8eFBuuukmefjhh+XNN9+UjRs3SkJCgixevFjuvfdeee+99+zzW1pabEClpqbK+++/L+fOnZMf/ehHEhYWJk8//XRPfUb0IFNBl75ggXz+9NPScOZMl17LlIrHjBkjMTfcIEnf/rYNKir0gP7N5Zi06YKkpCR55pln5Ac/+IEMGjRIXn31VfuzcezYMRk9erQUFhbKjBkzbK/re9/7npw9e1ZS/n6dtJdeekl+/vOfy/nz5yU8PLxD71ldXW1D0OPx2B4d+pa5FFJVUZGceflle9+oTgkJsUGUPHOmJP/Hf9grQ4QmJBBOQJCr7uD3+DUfkzK9ItNjqqurs8N+xcXF0tTUJFlZWb5tRo0aJRkZGb6QMvNx48b5AsrIzs6WhQsXSklJiUycOLHd92poaLBT2w8HXWXgidOmSWhsrJxZt04u/+1v4jQ1fe1zzB1tTSGEuX6eKYIIS0iww3xU6AHoUkgdPnzYhpI5/mSOO23atEnGjBkjhw4dsj2hxMREv+1NIJWVldmfzbxtQHnXe9d9ldzcXHnyySc7u6voRabnY67yMPyRR+Tinj1S89FHcuXECXs/KXPbi7CkJHvSrxnSs1cdHz3aVunZi7sSTAC6K6RuvPFGG0imi/baa69JTk6OFPTwxUVXrlwpy5cv9+tJpaen9+h7ovNM2JjhupQ5c2Tgd79rr47u7VGZmwmakWV3aKiEJSeLu4NDuwD6t06HlOktjRgxwv48efJkKSoqkueee07uu+8+aWxslKqqKr/elKnuM4UShpnv27fP7/W81X/ebdoTERFhJwQGE0TupKQve08A0AVdPjrd2tpqjxeZwDJVevn5+b51paWltuTcDA8aZm6GCysqKnzb5OXl2YNmZsgQAIBr7kmZYbc77rjDFkPU1NTYSr5du3bJ22+/bas05s+fb4flTMWfCZ4lS5bYYDJFE8asWbNsGM2bN0/WrFljj0M99thj9twqekoAgC6FlOkBmfOazPlNJpTMib0moG677Ta7/tlnnxW3221P4jW9K1O59+KLL/qeHxISIlu3brXVfCa8YmJi7DGtp556qjO7AQDoJ7p8nlRf4DwpAAhsHf0e54xJAIBahBQAQC1CCgCgFiEFAFCLkAIAqEVIAQDUIqQAAGoRUgAAtQgpAIBahBQAQC1CCgCgFiEFAFCLkAIAqEVIAQDUIqQAAGoRUgAAtQgpAIBahBQAQC1CCgCgFiEFAFCLkAIAqEVIAQDUIqQAAGoRUgAAtQgpAIBahBQAQC1CCgCgFiEFAFCLkAIAqEVIAQDUIqQAAGoRUgAAtQgpAIBahBQAQC1CCgCgFiEFAFCLkAIAqEVIAQDUIqQAAGoRUgAAtQgpAIBahBQAQC1CCgCgFiEFAFCLkAIAqEVIAQDUIqQAAGoRUgAAtQgpAIBahBQAQC1CCgCgFiEFAFCLkAIAqEVIAQDUIqQAAGoRUgAAtQgpAIBahBQAQC1CCgCgFiEFAFCLkAIAqEVIAQDUIqQAAMEZUqtXrxaXyyXLli3zLauvr5dFixZJcnKyxMbGyty5c6W8vNzveadOnZLZs2dLdHS0DB48WFasWCHNzc1d2RUAQBC65pAqKiqSP/zhDzJ+/Hi/5Q8//LBs2bJFNm7cKAUFBXL27Fm59957fetbWlpsQDU2Nsr7778v69atk1deeUVWrVrVtU8CAAg+zjWoqalxRo4c6eTl5Tm33nqrs3TpUru8qqrKCQsLczZu3Ojb9ujRo455m8LCQvt427ZtjtvtdsrKynzbrF271omPj3caGho69P4ej8e+ppkDAAJPR7/Hr6knZYbzTG8oKyvLb3lxcbE0NTX5LR81apRkZGRIYWGhfWzm48aNk5SUFN822dnZUl1dLSUlJe2+X0NDg13fdgIABL/Qzj5hw4YNcuDAATvcd7WysjIJDw+XxMREv+UmkMw67zZtA8q73ruuPbm5ufLkk092dlcBAAGuUz2p06dPy9KlS+XPf/6zREZGSm9ZuXKleDwe32T2AwAQ/DoVUmY4r6KiQiZNmiShoaF2MsURzz//vP3Z9IhMQURVVZXf80x1X2pqqv3ZzK+u9vM+9m5ztYiICImPj/ebAADBr1MhNXPmTDl8+LAcOnTIN02ZMkUeeOAB389hYWGSn5/ve05paaktOc/MzLSPzdy8hgk7r7y8PBs8Y8aM6c7PBgDoT8ek4uLiZOzYsX7LYmJi7DlR3uXz58+X5cuXS1JSkg2eJUuW2GCaMWOGXT9r1iwbRvPmzZM1a9bY41CPPfaYLcYwPSYAAK65cOKbPPvss+J2u+1JvKYqz1Tuvfjii771ISEhsnXrVlm4cKENLxNyOTk58tRTT3X3rgAAApzL1KFLgDEl6AkJCbaIguNTABB4Ovo9zrX7AABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEA1CKkAABqhUoAchzHzqurq/t6VwAA18D7/e39Pg+qkLpw4YKdp6en9/WuAAC6oKamRhISEoIrpJKSkuz81KlTX/vh+jvzLxUT5KdPn5b4+Pi+3h21aKeOoZ06hnbqGNODMgGVlpb2tdsFZEi53V8eSjMBxS/BNzNtRDt9M9qpY2injqGdvllHOhkUTgAA1CKkAABqBWRIRUREyBNPPGHn+Gq0U8fQTh1DO3UM7dS9XM431f8BANBHArInBQDoHwgpAIBahBQAQC1CCgCgVkCG1AsvvCDDhg2TyMhImT59uuzbt0/6k927d8tdd91lz9R2uVyyefNmv/WmFmbVqlUyZMgQiYqKkqysLPn000/9trl48aI88MAD9mTDxMREmT9/vtTW1kqwyM3NlalTp0pcXJwMHjxY5syZI6WlpX7b1NfXy6JFiyQ5OVliY2Nl7ty5Ul5e7reNuarJ7NmzJTo62r7OihUrpLm5WYLF2rVrZfz48b4TTzMzM2X79u2+9bRR+1avXm3/9pYtW+ZbRlv1ECfAbNiwwQkPD3f+9Kc/OSUlJc5DDz3kJCYmOuXl5U5/sW3bNueXv/yl8/rrr5vKTGfTpk1+61evXu0kJCQ4mzdvdj788EPn+9//vjN8+HDnypUrvm1uv/12Z8KECc4HH3zgvPvuu86IESOc+++/3wkW2dnZzssvv+wcOXLEOXTokHPnnXc6GRkZTm1trW+bn/70p056erqTn5/v7N+/35kxY4Zz8803+9Y3Nzc7Y8eOdbKyspyDBw/adh84cKCzcuVKJ1j89a9/dd58803nk08+cUpLS51f/OIXTlhYmG03gzb6Z/v27XOGDRvmjB8/3lm6dKlvOW3VMwIupKZNm+YsWrTI97ilpcVJS0tzcnNznf7o6pBqbW11UlNTnWeeeca3rKqqyomIiHDWr19vH3/88cf2eUVFRb5ttm/f7rhcLufMmTNOMKqoqLCfuaCgwNcm5st448aNvm2OHj1qtyksLLSPzZeI2+12ysrKfNusXbvWiY+PdxoaGpxgNWDAAOePf/wjbdSOmpoaZ+TIkU5eXp5z6623+kKKtuo5ATXc19jYKMXFxXb4qu11/MzjwsLCPt03LY4fPy5lZWV+bWSuj2WGRb1tZOZmiG/KlCm+bcz2pi337t0rwcjj8fhdnNj8HjU1Nfm106hRoyQjI8OvncaNGycpKSm+bbKzs+0FREtKSiTYtLS0yIYNG6Surs4O+9FG/8wM55nhurZtYtBWPSegLjBbWVlp/5Da/k82zONjx4712X5pYgLKaK+NvOvM3IyHtxUaGmq/wL3bBJPW1lZ77OCWW26RsWPH2mXmc4aHh9uw/rp2aq8dveuCxeHDh20omWMq5ljKpk2bZMyYMXLo0CHaqA0T4AcOHJCioqJ/WsfvU88JqJACrvVfv0eOHJE9e/b09a6odOONN9pAMr3N1157TXJycqSgoKCvd0sVc9uNpUuXSl5eni3YQu8JqOG+gQMHSkhIyD9VzJjHqampfbZfmnjb4evayMwrKir81psKI1PxF2ztuHjxYtm6davs3LlThg4d6ltuPqcZPq6qqvradmqvHb3rgoXpAYwYMUImT55sqyInTJggzz33HG101XCe+ZuZNGmSHXUwkwny559/3v5sekS0Vc9wB9ofk/lDys/P9xvKMY/NcAVEhg8fbn/h27aRGfM2x5q8bWTm5o/J/OF57dixw7alOXYVDExNiQkoM3RlPptpl7bM71FYWJhfO5kSdVMi3LadzFBY20A3/5I2pdpmOCxYmd+DhoYG2qiNmTNn2s9pepzeyRzTNadxeH+mrXqIE4Al6KZS7ZVXXrFVagsWLLAl6G0rZoKdqTAyJaxmMv8Lf/Ob39ifT5486StBN23yxhtvOB999JFz9913t1uCPnHiRGfv3r3Onj17bMVSMJWgL1y40Jbh79q1yzl37pxvunz5sl/JsClL37Fjhy0ZzszMtNPVJcOzZs2yZexvvfWWM2jQoKAqGX700UdtxePx48ft74p5bKo833nnHbueNvpqbav7DNqqZwRcSBm/+93v7C+DOV/KlKSbc336k507d9pwunrKycnxlaE//vjjTkpKig30mTNn2nNg2rpw4YINpdjYWFsC++CDD9rwCxbttY+ZzLlTXia0f/azn9mS6+joaOeee+6xQdbWiRMnnDvuuMOJioqy57Q88sgjTlNTkxMsfvKTnzjXXXed/VsyX5jmd8UbUAZt1PGQoq16BrfqAACoFVDHpAAA/QshBQBQi5ACAKhFSAEA1CKkAABqEVIAALUIKQCAWoQUAEAtQgoAoBYhBQBQi5ACAKhFSAEARKv/Dzt9QdxLuhxxAAAAAElFTkSuQmCC"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-02T12:25:41.493774Z",
     "start_time": "2025-09-02T12:25:40.496850Z"
    }
   },
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "#DuelingDQN和其他DQN模型不同的点,它使用的是不同的模型结构\n",
    "class VAnet(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "        self.fc = torch.nn.Sequential(\n",
    "            torch.nn.Linear(3, 128),\n",
    "            torch.nn.ReLU(),\n",
    "        )\n",
    "\n",
    "        self.fc_A = torch.nn.Linear(128, 11)\n",
    "        self.fc_V = torch.nn.Linear(128, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        #[5, 11] -> [5, 128] -> [5, 11]\n",
    "        A = self.fc_A(self.fc(x))\n",
    "\n",
    "        #[5, 11] -> [5, 128] -> [5, 1]\n",
    "        V = self.fc_V(self.fc(x))\n",
    "\n",
    "        #[5, 11] -> [5] -> [5, 1]\n",
    "        A_mean = A.mean(dim=1).reshape(-1, 1)\n",
    "\n",
    "        #[5, 11] - [5, 1] = [5, 11]\n",
    "        A -= A_mean\n",
    "\n",
    "        #Q值由V值和A值计算得到\n",
    "        #[5, 11] + [5, 1] = [5, 11]\n",
    "        Q = A + V\n",
    "\n",
    "        return Q\n",
    "\n",
    "\n",
    "VAnet()(torch.randn(5, 3)).shape"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 11])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-02T12:25:41.503427Z",
     "start_time": "2025-09-02T12:25:41.500590Z"
    }
   },
   "source": [
    "import torch\n",
    "\n",
    "#计算动作的模型,也是真正要用的模型\n",
    "model = VAnet()\n",
    "\n",
    "#经验网络,用于评估一个状态的分数\n",
    "next_model = VAnet()\n",
    "\n",
    "#把model的参数复制给next_model\n",
    "next_model.load_state_dict(model.state_dict())\n",
    "\n",
    "model, next_model"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(VAnet(\n",
       "   (fc): Sequential(\n",
       "     (0): Linear(in_features=3, out_features=128, bias=True)\n",
       "     (1): ReLU()\n",
       "   )\n",
       "   (fc_A): Linear(in_features=128, out_features=11, bias=True)\n",
       "   (fc_V): Linear(in_features=128, out_features=1, bias=True)\n",
       " ),\n",
       " VAnet(\n",
       "   (fc): Sequential(\n",
       "     (0): Linear(in_features=3, out_features=128, bias=True)\n",
       "     (1): ReLU()\n",
       "   )\n",
       "   (fc_A): Linear(in_features=128, out_features=11, bias=True)\n",
       "   (fc_V): Linear(in_features=128, out_features=1, bias=True)\n",
       " ))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-02T12:25:41.518079Z",
     "start_time": "2025-09-02T12:25:41.515038Z"
    }
   },
   "source": [
    "import random\n",
    "\n",
    "\n",
    "def get_action(state):\n",
    "    #走神经网络,得到一个动作\n",
    "    state = torch.FloatTensor(state).reshape(1, 3)\n",
    "    action = model(state).argmax().item()\n",
    "\n",
    "    if random.random() < 0.01:\n",
    "        action = random.choice(range(11))\n",
    "\n",
    "    #离散动作连续化\n",
    "    action_continuous = action\n",
    "    action_continuous /= 10\n",
    "    action_continuous *= 4\n",
    "    action_continuous -= 2\n",
    "\n",
    "    return action, action_continuous\n",
    "\n",
    "\n",
    "get_action([0.29292667, 0.9561349, 1.0957013])"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9, 1.6)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 5
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-02T12:25:41.560274Z",
     "start_time": "2025-09-02T12:25:41.545049Z"
    }
   },
   "source": [
    "#样本池\n",
    "datas = []\n",
    "\n",
    "\n",
    "#向样本池中添加N条数据,删除M条最古老的数据\n",
    "def update_data():\n",
    "    old_count = len(datas)\n",
    "\n",
    "    #玩到新增了N个数据为止\n",
    "    while len(datas) - old_count < 200:\n",
    "        #初始化游戏\n",
    "        state = env.reset()\n",
    "\n",
    "        #玩到游戏结束为止\n",
    "        over = False\n",
    "        while not over:\n",
    "            #根据当前状态得到一个动作\n",
    "            action, action_continuous = get_action(state)\n",
    "\n",
    "            #执行动作,得到反馈\n",
    "            next_state, reward, over, _ = env.step([action_continuous])\n",
    "\n",
    "            #记录数据样本\n",
    "            datas.append((state, action, reward, next_state, over))\n",
    "\n",
    "            #更新游戏状态,开始下一个动作\n",
    "            state = next_state\n",
    "\n",
    "    update_count = len(datas) - old_count\n",
    "    drop_count = max(len(datas) - 5000, 0)\n",
    "\n",
    "    #数据上限,超出时从最古老的开始删除\n",
    "    while len(datas) > 5000:\n",
    "        datas.pop(0)\n",
    "\n",
    "    return update_count, drop_count\n",
    "\n",
    "\n",
    "update_data(), len(datas)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((200, 0), 200)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 6
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-02T12:25:41.575819Z",
     "start_time": "2025-09-02T12:25:41.567921Z"
    }
   },
   "source": [
    "#获取一批数据样本\n",
    "def get_sample():\n",
    "    #从样本池中采样\n",
    "    samples = random.sample(datas, 64)\n",
    "\n",
    "    #[b, 3]\n",
    "    state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 3)\n",
    "    #[b, 1]\n",
    "    action = torch.LongTensor([i[1] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 1]\n",
    "    reward = torch.FloatTensor([i[2] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 3]\n",
    "    next_state = torch.FloatTensor([i[3] for i in samples]).reshape(-1, 3)\n",
    "    #[b, 1]\n",
    "    over = torch.LongTensor([i[4] for i in samples]).reshape(-1, 1)\n",
    "\n",
    "    return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "state, action, reward, next_state, over"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/s9/yht5_svd6mxft5fpm7ht48f80000gn/T/ipykernel_92463/3503280997.py:7: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/torch/csrc/utils/tensor_new.cpp:256.)\n",
      "  state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 3)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[-3.2764e-01,  9.4480e-01,  1.3821e+00],\n",
       "         [-2.4138e-01,  9.7043e-01, -5.4956e-01],\n",
       "         [-7.2500e-01,  6.8875e-01,  5.2685e+00],\n",
       "         [-9.4619e-01, -3.2362e-01,  3.3746e+00],\n",
       "         [-8.5343e-01,  5.2120e-01,  2.4920e+00],\n",
       "         [-6.5032e-01, -7.5966e-01,  2.0585e+00],\n",
       "         [-5.1641e-01, -8.5634e-01, -3.1460e+00],\n",
       "         [-9.8588e-01, -1.6747e-01, -4.0935e+00],\n",
       "         [-5.6572e-01, -8.2460e-01, -7.1339e-01],\n",
       "         [-3.4567e-01, -9.3836e-01,  6.1019e-01],\n",
       "         [-7.6249e-01, -6.4700e-01, -1.2822e+00],\n",
       "         [-7.3589e-01, -6.7710e-01, -3.0296e+00],\n",
       "         [-7.9161e-01,  6.1102e-01, -5.3125e+00],\n",
       "         [-8.4354e-01, -5.3706e-01, -3.5374e+00],\n",
       "         [-9.2131e-01, -3.8883e-01,  4.0234e+00],\n",
       "         [-6.7801e-01, -7.3505e-01,  1.5261e-01],\n",
       "         [-9.0704e-01,  4.2105e-01, -3.8288e+00],\n",
       "         [-9.7811e-01,  2.0811e-01,  3.6495e+00],\n",
       "         [-9.4641e-01,  3.2298e-01,  2.5105e+00],\n",
       "         [-7.9299e-01,  6.0924e-01, -2.2295e+00],\n",
       "         [-3.3423e-02,  9.9944e-01,  1.4677e+00],\n",
       "         [-9.9208e-01, -1.2557e-01, -2.7548e+00],\n",
       "         [-8.3152e-01, -5.5549e-01,  3.7918e+00],\n",
       "         [-8.5674e-01, -5.1575e-01,  6.6576e+00],\n",
       "         [-8.6619e-01, -4.9971e-01, -1.3830e+00],\n",
       "         [-9.9998e-01, -5.7496e-03, -2.2899e+00],\n",
       "         [-9.8538e-02, -9.9513e-01,  3.3809e+00],\n",
       "         [-1.7260e-01, -9.8499e-01, -1.1080e+00],\n",
       "         [-9.9514e-01,  9.8484e-02,  2.6238e+00],\n",
       "         [-9.7912e-01, -2.0329e-01,  4.8589e+00],\n",
       "         [-4.3113e-01, -9.0229e-01,  1.8624e+00],\n",
       "         [-8.1568e-01,  5.7850e-01,  6.3453e-01],\n",
       "         [-8.4512e-01,  5.3457e-01, -1.5248e+00],\n",
       "         [-5.3002e-01, -8.4798e-01,  4.3578e-01],\n",
       "         [-9.2455e-01,  3.8106e-01, -2.2821e+00],\n",
       "         [-7.2671e-02, -9.9736e-01, -2.3791e+00],\n",
       "         [-9.7824e-01, -2.0748e-01,  6.7532e+00],\n",
       "         [-6.9811e-01,  7.1599e-01, -1.1464e+00],\n",
       "         [-9.7788e-01,  2.0915e-01, -2.0949e+00],\n",
       "         [-8.1305e-01, -5.8219e-01, -4.3461e+00],\n",
       "         [ 2.0101e-01, -9.7959e-01,  1.3220e+00],\n",
       "         [-9.9997e-01,  7.3632e-03,  4.0456e+00],\n",
       "         [-7.9993e-01, -6.0010e-01, -6.3411e-01],\n",
       "         [-7.2339e-01,  6.9043e-01,  1.0267e+00],\n",
       "         [-6.8955e-01,  7.2424e-01, -4.1260e+00],\n",
       "         [ 3.9947e-02,  9.9920e-01,  4.7834e-01],\n",
       "         [-2.4822e-01, -9.6870e-01,  2.3542e+00],\n",
       "         [-9.9563e-01,  9.3435e-02, -3.4176e+00],\n",
       "         [-7.6395e-01, -6.4527e-01, -2.2883e+00],\n",
       "         [-7.2952e-01,  6.8396e-01,  4.0996e+00],\n",
       "         [ 1.3586e-01, -9.9073e-01,  2.0051e+00],\n",
       "         [-8.5176e-01,  5.2394e-01,  1.3084e+00],\n",
       "         [-7.9692e-01,  6.0408e-01, -5.8537e-02],\n",
       "         [-9.8042e-01, -1.9690e-01,  4.1111e+00],\n",
       "         [-9.8408e-01, -1.7770e-01,  2.7882e+00],\n",
       "         [-6.9036e-01, -7.2347e-01, -3.3868e-01],\n",
       "         [-7.3798e-01,  6.7482e-01, -1.7125e+00],\n",
       "         [-7.8204e-01,  6.2323e-01,  1.7846e+00],\n",
       "         [-8.7062e-01,  4.9195e-01, -2.6621e-02],\n",
       "         [-9.9991e-01,  1.3540e-02, -2.7890e+00],\n",
       "         [-9.9439e-01,  1.0576e-01, -2.2343e+00],\n",
       "         [-9.2781e-01, -3.7304e-01, -4.7827e+00],\n",
       "         [-3.7589e-01, -9.2666e-01, -2.4510e+00],\n",
       "         [-9.9889e-01, -4.7162e-02,  5.6955e+00]]),\n",
       " tensor([[9],\n",
       "         [9],\n",
       "         [9],\n",
       "         [6],\n",
       "         [9],\n",
       "         [6],\n",
       "         [5],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [5],\n",
       "         [5],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [9],\n",
       "         [9],\n",
       "         [6],\n",
       "         [9],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [9],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [8],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [5],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [9],\n",
       "         [6],\n",
       "         [9],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [9],\n",
       "         [6],\n",
       "         [9],\n",
       "         [9],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [9],\n",
       "         [9],\n",
       "         [6],\n",
       "         [6],\n",
       "         [5],\n",
       "         [5],\n",
       "         [6]]),\n",
       " tensor([[ -3.8211],\n",
       "         [ -3.3255],\n",
       "         [ -8.4514],\n",
       "         [ -9.0465],\n",
       "         [ -7.3489],\n",
       "         [ -5.6168],\n",
       "         [ -5.4564],\n",
       "         [-10.5165],\n",
       "         [ -4.7691],\n",
       "         [ -3.7382],\n",
       "         [ -6.1082],\n",
       "         [ -6.6672],\n",
       "         [ -8.9937],\n",
       "         [ -7.8803],\n",
       "         [ -9.1388],\n",
       "         [ -5.3656],\n",
       "         [ -8.7939],\n",
       "         [ -9.9308],\n",
       "         [ -8.5442],\n",
       "         [ -6.6799],\n",
       "         [ -2.7915],\n",
       "         [ -9.8534],\n",
       "         [ -7.9539],\n",
       "         [-11.1910],\n",
       "         [ -7.0470],\n",
       "         [-10.3581],\n",
       "         [ -3.9304],\n",
       "         [ -3.1654],\n",
       "         [ -9.9481],\n",
       "         [-10.9862],\n",
       "         [ -4.4135],\n",
       "         [ -6.4170],\n",
       "         [ -6.8767],\n",
       "         [ -4.5536],\n",
       "         [ -8.0870],\n",
       "         [ -3.2687],\n",
       "         [-13.1609],\n",
       "         [ -5.6238],\n",
       "         [ -9.0291],\n",
       "         [ -8.2401],\n",
       "         [ -2.0475],\n",
       "         [-11.4602],\n",
       "         [ -6.2802],\n",
       "         [ -5.7700],\n",
       "         [ -7.1392],\n",
       "         [ -2.3689],\n",
       "         [ -3.8727],\n",
       "         [-10.4586],\n",
       "         [ -6.4784],\n",
       "         [ -7.3877],\n",
       "         [ -2.4600],\n",
       "         [ -6.8825],\n",
       "         [ -6.2178],\n",
       "         [-10.3539],\n",
       "         [ -9.5566],\n",
       "         [ -5.4535],\n",
       "         [ -6.0576],\n",
       "         [ -6.4157],\n",
       "         [ -6.9052],\n",
       "         [-10.5627],\n",
       "         [ -9.7144],\n",
       "         [ -9.9012],\n",
       "         [ -4.4273],\n",
       "         [-12.8195]]),\n",
       " tensor([[-0.4353,  0.9003,  2.3307],\n",
       "         [-0.2616,  0.9652,  0.4183],\n",
       "         [-0.8967,  0.4426,  6.0250],\n",
       "         [-0.8827, -0.4699,  3.1919],\n",
       "         [-0.9241,  0.3821,  3.1229],\n",
       "         [-0.5896, -0.8077,  1.5487],\n",
       "         [-0.6684, -0.7438, -3.7882],\n",
       "         [-0.9992,  0.0397, -4.1591],\n",
       "         [-0.6170, -0.7870, -1.2718],\n",
       "         [-0.3472, -0.9378, -0.0336],\n",
       "         [-0.8149, -0.5796, -1.7074],\n",
       "         [-0.8435, -0.5371, -3.5374],\n",
       "         [-0.6216,  0.7834, -4.8543],\n",
       "         [-0.9313, -0.3644, -3.8802],\n",
       "         [-0.8315, -0.5555,  3.7918],\n",
       "         [-0.6904, -0.7235, -0.3387],\n",
       "         [-0.8212,  0.5706, -3.4530],\n",
       "         [-1.0000,  0.0074,  4.0456],\n",
       "         [-0.9840,  0.1783,  2.9928],\n",
       "         [-0.7380,  0.6748, -1.7125],\n",
       "         [-0.1557,  0.9878,  2.4573],\n",
       "         [-0.9999,  0.0135, -2.7890],\n",
       "         [-0.7243, -0.6894,  3.4352],\n",
       "         [-0.6536, -0.7568,  6.3308],\n",
       "         [-0.9054, -0.4245, -1.6977],\n",
       "         [-0.9944,  0.1058, -2.2343],\n",
       "         [ 0.0360, -0.9994,  2.6946],\n",
       "         [-0.2598, -0.9657, -1.7867],\n",
       "         [-0.9992, -0.0392,  2.7576],\n",
       "         [-0.9035, -0.4287,  4.7664],\n",
       "         [-0.3741, -0.9274,  1.2457],\n",
       "         [-0.8518,  0.5239,  1.3084],\n",
       "         [-0.8155,  0.5788, -1.0639],\n",
       "         [-0.5360, -0.8442, -0.1402],\n",
       "         [-0.8834,  0.4686, -1.9363],\n",
       "         [-0.2183, -0.9759, -2.9471],\n",
       "         [-0.8567, -0.5157,  6.6576],\n",
       "         [-0.6782,  0.7349, -0.5494],\n",
       "         [-0.9540,  0.2999, -1.8781],\n",
       "         [-0.9278, -0.3730, -4.7827],\n",
       "         [ 0.2326, -0.9726,  0.6473],\n",
       "         [-0.9804, -0.1969,  4.1111],\n",
       "         [-0.8296, -0.5584, -1.0242],\n",
       "         [-0.7820,  0.6232,  1.7846],\n",
       "         [-0.5520,  0.8339, -3.5228],\n",
       "         [-0.0334,  0.9994,  1.4677],\n",
       "         [-0.1657, -0.9862,  1.6876],\n",
       "         [-0.9669,  0.2551, -3.2875],\n",
       "         [-0.8442, -0.5361, -2.7122],\n",
       "         [-0.8725,  0.4887,  4.8525],\n",
       "         [ 0.2010, -0.9796,  1.3220],\n",
       "         [-0.8985,  0.4389,  1.9414],\n",
       "         [-0.8157,  0.5785,  0.6345],\n",
       "         [-0.9213, -0.3888,  4.0234],\n",
       "         [-0.9510, -0.3092,  2.7150],\n",
       "         [-0.7195, -0.6945, -0.8213],\n",
       "         [-0.6981,  0.7160, -1.1464],\n",
       "         [-0.8534,  0.5212,  2.4920],\n",
       "         [-0.8846,  0.4664,  0.5823],\n",
       "         [-0.9888,  0.1489, -2.7188],\n",
       "         [-0.9779,  0.2092, -2.0949],\n",
       "         [-0.9917, -0.1288, -5.0625],\n",
       "         [-0.5164, -0.8563, -3.1460],\n",
       "         [-0.9450, -0.3271,  5.7202]]),\n",
       " tensor([[0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0]]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 7
  },
  {
   "cell_type": "code",
   "metadata": {
    "scrolled": true,
    "ExecuteTime": {
     "end_time": "2025-09-02T12:25:41.672574Z",
     "start_time": "2025-09-02T12:25:41.669686Z"
    }
   },
   "source": [
    "def get_value(state, action):\n",
    "    #使用状态计算出动作的logits\n",
    "    #[b, 3] -> [b, 11]\n",
    "    value = model(state)\n",
    "\n",
    "    #根据实际使用的action取出每一个值\n",
    "    #这个值就是模型评估的在该状态下,执行动作的分数\n",
    "    #在执行动作前,显然并不知道会得到的反馈和next_state\n",
    "    #所以这里不能也不需要考虑next_state和reward\n",
    "    #[b, 11] -> [b, 1]\n",
    "    value = value.gather(dim=1, index=action)\n",
    "\n",
    "    return value\n",
    "\n",
    "\n",
    "get_value(state, action)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.9553],\n",
       "        [0.6030],\n",
       "        [1.3505],\n",
       "        [0.7050],\n",
       "        [0.8765],\n",
       "        [0.5095],\n",
       "        [0.8878],\n",
       "        [1.2393],\n",
       "        [0.5351],\n",
       "        [0.3404],\n",
       "        [0.6722],\n",
       "        [0.9211],\n",
       "        [1.4849],\n",
       "        [1.0510],\n",
       "        [0.7801],\n",
       "        [0.4710],\n",
       "        [1.1897],\n",
       "        [0.8378],\n",
       "        [0.7751],\n",
       "        [0.8340],\n",
       "        [1.0550],\n",
       "        [0.9839],\n",
       "        [0.7261],\n",
       "        [1.1453],\n",
       "        [0.7308],\n",
       "        [0.9026],\n",
       "        [0.6415],\n",
       "        [0.3931],\n",
       "        [0.6858],\n",
       "        [0.9304],\n",
       "        [0.4429],\n",
       "        [0.6909],\n",
       "        [0.7312],\n",
       "        [0.3833],\n",
       "        [0.8827],\n",
       "        [0.6031],\n",
       "        [1.2025],\n",
       "        [0.6604],\n",
       "        [0.8612],\n",
       "        [1.2166],\n",
       "        [0.2899],\n",
       "        [0.8520],\n",
       "        [0.6585],\n",
       "        [0.7692],\n",
       "        [1.2069],\n",
       "        [0.8902],\n",
       "        [0.4686],\n",
       "        [1.1182],\n",
       "        [0.8041],\n",
       "        [1.1701],\n",
       "        [0.3944],\n",
       "        [0.7061],\n",
       "        [0.6859],\n",
       "        [0.8229],\n",
       "        [0.6549],\n",
       "        [0.5424],\n",
       "        [0.7251],\n",
       "        [0.8334],\n",
       "        [0.6783],\n",
       "        [0.9940],\n",
       "        [0.8910],\n",
       "        [1.3439],\n",
       "        [0.6807],\n",
       "        [1.0805]], grad_fn=<GatherBackward0>)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 8
  },
  {
   "cell_type": "code",
   "metadata": {
    "scrolled": true,
    "ExecuteTime": {
     "end_time": "2025-09-02T12:25:41.772075Z",
     "start_time": "2025-09-02T12:25:41.768036Z"
    }
   },
   "source": [
    "def get_target(reward, next_state, over):\n",
    "    #上面已经把模型认为的状态下执行动作的分数给评估出来了\n",
    "    #下面使用next_state和reward计算真实的分数\n",
    "    #针对一个状态,它到底应该多少分,可以使用以往模型积累的经验评估\n",
    "    #这也是没办法的办法,因为显然没有精确解,这里使用延迟更新的next_model评估\n",
    "\n",
    "    #使用next_state计算下一个状态的分数\n",
    "    #[b, 3] -> [b, 11]\n",
    "    with torch.no_grad():\n",
    "        target = next_model(next_state)\n",
    "\n",
    "    #取所有动作中分数最大的\n",
    "    #[b, 11] -> [b, 1]\n",
    "    target = target.max(dim=1)[0]\n",
    "    target = target.reshape(-1, 1)\n",
    "\n",
    "    #下一个状态的分数乘以一个系数,相当于权重\n",
    "    target *= 0.98\n",
    "\n",
    "    #如果next_state已经游戏结束,则next_state的分数是0\n",
    "    #因为如果下一步已经游戏结束,显然不需要再继续玩下去,也就不需要考虑next_state了.\n",
    "    #[b, 1] * [b, 1] -> [b, 1]\n",
    "    target *= (1 - over)\n",
    "\n",
    "    #加上reward就是最终的分数\n",
    "    #[b, 1] + [b, 1] -> [b, 1]\n",
    "    target += reward\n",
    "\n",
    "    return target\n",
    "\n",
    "\n",
    "get_target(reward, next_state, over)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ -2.7587],\n",
       "        [ -2.5050],\n",
       "        [ -7.1800],\n",
       "        [ -8.4006],\n",
       "        [ -6.4904],\n",
       "        [ -5.1622],\n",
       "        [ -4.4115],\n",
       "        [ -9.2746],\n",
       "        [ -4.1769],\n",
       "        [ -3.4028],\n",
       "        [ -5.3855],\n",
       "        [ -5.6372],\n",
       "        [ -7.6679],\n",
       "        [ -6.7447],\n",
       "        [ -8.4272],\n",
       "        [ -4.8341],\n",
       "        [ -7.7245],\n",
       "        [ -9.0958],\n",
       "        [ -7.8108],\n",
       "        [ -5.9693],\n",
       "        [ -1.6419],\n",
       "        [ -8.8793],\n",
       "        [ -7.3012],\n",
       "        [-11.1910],\n",
       "        [ -6.2801],\n",
       "        [ -9.4849],\n",
       "        [ -3.4267],\n",
       "        [ -2.6639],\n",
       "        [ -9.2853],\n",
       "        [-10.1216],\n",
       "        [ -4.0326],\n",
       "        [ -5.7249],\n",
       "        [ -6.2053],\n",
       "        [ -4.1270],\n",
       "        [ -7.2975],\n",
       "        [ -2.5328],\n",
       "        [-12.0384],\n",
       "        [ -4.9853],\n",
       "        [ -8.2267],\n",
       "        [ -6.9231],\n",
       "        [ -1.8026],\n",
       "        [-10.6538],\n",
       "        [ -5.5952],\n",
       "        [ -4.9533],\n",
       "        [ -6.1170],\n",
       "        [ -1.3350],\n",
       "        [ -3.4996],\n",
       "        [ -9.3891],\n",
       "        [ -5.5831],\n",
       "        [ -6.2536],\n",
       "        [ -2.1759],\n",
       "        [ -6.1298],\n",
       "        [ -5.5407],\n",
       "        [ -9.5894],\n",
       "        [ -8.9438],\n",
       "        [ -4.8348],\n",
       "        [ -5.4104],\n",
       "        [ -5.5567],\n",
       "        [ -6.2631],\n",
       "        [ -9.6012],\n",
       "        [ -8.8704],\n",
       "        [ -8.4878],\n",
       "        [ -3.5573],\n",
       "        [-11.8044]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 9
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-02T12:25:41.850379Z",
     "start_time": "2025-09-02T12:25:41.836334Z"
    }
   },
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play):\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #记录反馈值的和,这个值越大越好\n",
    "    reward_sum = 0\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        _, action_continuous = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        state, reward, over, _ = env.step([action_continuous])\n",
    "        reward_sum += reward\n",
    "\n",
    "        #打印动画\n",
    "        if play and random.random() < 0.2:  #跳帧\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "test(play=False)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1823.8748559024018"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 10
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "OHoSU6uI-xIt",
    "scrolled": false,
    "ExecuteTime": {
     "end_time": "2025-09-02T12:25:48.217752Z",
     "start_time": "2025-09-02T12:25:41.879228Z"
    }
   },
   "source": [
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=1e-2)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #训练N次\n",
    "    for epoch in range(200):\n",
    "        #更新N条数据\n",
    "        update_count, drop_count = update_data()\n",
    "\n",
    "        #每次更新过数据后,学习N次\n",
    "        for i in range(200):\n",
    "            #采样一批数据\n",
    "            state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "            #计算一批样本的value和target\n",
    "            value = get_value(state, action)\n",
    "            target = get_target(reward, next_state, over)\n",
    "\n",
    "            #更新参数\n",
    "            loss = loss_fn(value, target)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            #把model的参数复制给next_model\n",
    "            if (i + 1) % 50 == 0:\n",
    "                next_model.load_state_dict(model.state_dict())\n",
    "\n",
    "        if epoch % 20 == 0:\n",
    "            test_result = sum([test(play=False) for _ in range(20)]) / 20\n",
    "            print(epoch, len(datas), update_count, drop_count, test_result)\n",
    "\n",
    "\n",
    "train()"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 400 200 0 -1263.912429765591\n",
      "20 4400 200 0 -856.8865260371888\n",
      "40 5000 200 200 -1028.5538760655702\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[11], line 35\u001B[0m\n\u001B[1;32m     31\u001B[0m             test_result \u001B[38;5;241m=\u001B[39m \u001B[38;5;28msum\u001B[39m([test(play\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m) \u001B[38;5;28;01mfor\u001B[39;00m _ \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mrange\u001B[39m(\u001B[38;5;241m20\u001B[39m)]) \u001B[38;5;241m/\u001B[39m \u001B[38;5;241m20\u001B[39m\n\u001B[1;32m     32\u001B[0m             \u001B[38;5;28mprint\u001B[39m(epoch, \u001B[38;5;28mlen\u001B[39m(datas), update_count, drop_count, test_result)\n\u001B[0;32m---> 35\u001B[0m \u001B[43mtrain\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n",
      "Cell \u001B[0;32mIn[11], line 22\u001B[0m, in \u001B[0;36mtrain\u001B[0;34m()\u001B[0m\n\u001B[1;32m     20\u001B[0m \u001B[38;5;66;03m#更新参数\u001B[39;00m\n\u001B[1;32m     21\u001B[0m loss \u001B[38;5;241m=\u001B[39m loss_fn(value, target)\n\u001B[0;32m---> 22\u001B[0m \u001B[43moptimizer\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mzero_grad\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     23\u001B[0m loss\u001B[38;5;241m.\u001B[39mbackward()\n\u001B[1;32m     24\u001B[0m optimizer\u001B[38;5;241m.\u001B[39mstep()\n",
      "File \u001B[0;32m~/PycharmProjects/Simple_Reinforcement_Learning/.venv/lib/python3.10/site-packages/torch/_compile.py:41\u001B[0m, in \u001B[0;36m_disable_dynamo.<locals>.inner\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m     30\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m     31\u001B[0m \u001B[38;5;124;03mThis API should be only used inside torch, external users should still use\u001B[39;00m\n\u001B[1;32m     32\u001B[0m \u001B[38;5;124;03mtorch._dynamo.disable. The main goal of this API is to avoid circular\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m     37\u001B[0m \u001B[38;5;124;03mthe invocation of the decorated function.\u001B[39;00m\n\u001B[1;32m     38\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m     39\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m fn \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m---> 41\u001B[0m     \u001B[38;5;129m@functools\u001B[39m\u001B[38;5;241m.\u001B[39mwraps(fn)\n\u001B[1;32m     42\u001B[0m     \u001B[38;5;28;01mdef\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21minner\u001B[39m(\u001B[38;5;241m*\u001B[39margs: _P\u001B[38;5;241m.\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs: _P\u001B[38;5;241m.\u001B[39mkwargs) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m _T:\n\u001B[1;32m     43\u001B[0m         \u001B[38;5;66;03m# cache this on the first invocation to avoid adding too much overhead.\u001B[39;00m\n\u001B[1;32m     44\u001B[0m         disable_fn \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mgetattr\u001B[39m(fn, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m__dynamo_disable\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28;01mNone\u001B[39;00m)\n\u001B[1;32m     45\u001B[0m         \u001B[38;5;28;01mif\u001B[39;00m disable_fn \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "execution_count": 11
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyrklEQVR4nO3deXxTZb4/8M85aZIuIeneWGmhCIK1FGXP6LhRKVpBZzqOetmuol6Y4qjcy+uKCvOS66Uo4wJXFme8iuOM4MAFZBWZglWhbJVCgVIXlpZCWkpp0jVtk+f3B5IfRXBamuUJ+bxfr7xe5pwn53zzWPLJOefJcxQhhAAREZGEVH8XQEREdCUMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpKW30Jq4cKF6NmzJ0JDQzFs2DDs3r3bX6UQEZGk/BJSn3zyCaZNm4Y//OEP+OabbzBgwABkZmaiqqrKH+UQEZGkFH9MMDts2DAMGTIE77zzDgDA5XIhKSkJzzzzDF544QVfl0NERJIK8fUOW1paUFhYiBkzZriXqaqKjIwMFBQUXPY1DocDDofD/dzlcqGmpgYxMTFQFMXrNRMRkWcJIVBXV4fExESo6pVP6vk8pKqrq+F0OpGQkNBueUJCAo4cOXLZ1+Tm5uKVV17xRXlERORD5eXl6N69+xXX+zykrsaMGTMwbdo093ObzYbk5GSUl5fDaDT6sTIiIroadrsdSUlJ6Nat28+283lIxcbGQqPRoLKyst3yyspKmM3my75Gr9dDr9f/ZLnRaGRIEREFsH92ycbno/t0Oh0GDRqEvLw89zKXy4W8vDxYLBZfl0NERBLzy+m+adOmYeLEiRg8eDCGDh2Kt99+Gw0NDXj88cf9UQ4REUnKLyH1yCOP4MyZM5g1axasVituueUWfPbZZz8ZTEFERMHNL7+T6iq73Q6TyQSbzcZrUkREAaijn+Ocu4+IiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFqdDqkvv/wSo0ePRmJiIhRFwZo1a9qtF0Jg1qxZuO666xAWFoaMjAx899137drU1NRg7NixMBqNiIyMxKRJk1BfX9+lN0JERNeeTodUQ0MDBgwYgIULF152/euvv44FCxZgyZIl2LVrFyIiIpCZmYnm5mZ3m7Fjx+LQoUPYsmUL1q9fjy+//BJPP/301b8LIiK6NokuACBWr17tfu5yuYTZbBbz5s1zL6utrRV6vV4sW7ZMCCHE4cOHBQCxZ88ed5tNmzYJRVFERUVFh/Zrs9kEAGGz2bpSPhER+UlHP8c9ek3q2LFjsFqtyMjIcC8zmUwYNmwYCgoKAAAFBQWIjIzE4MGD3W0yMjKgqip27dp12e06HA7Y7fZ2DyIiuvZ5NKSsVisAICEhod3yhIQE9zqr1Yr4+Ph260NCQhAdHe1uc6nc3FyYTCb3IykpyZNlExGRpAJidN+MGTNgs9ncj/Lycn+XREREPuDRkDKbzQCAysrKdssrKyvd68xmM6qqqtqtb2trQ01NjbvNpfR6PYxGY7sHERFd+zwaUikpKTCbzcjLy3Mvs9vt2LVrFywWCwDAYrGgtrYWhYWF7jZbt26Fy+XCsGHDPFkOEREFuJDOvqC+vh7ff/+9+/mxY8dQVFSE6OhoJCcn47nnnsOrr76KPn36ICUlBTNnzkRiYiIeeughAMBNN92EUaNG4amnnsKSJUvQ2tqKqVOn4tFHH0ViYqLH3hgREV0DOjtscNu2bQLATx4TJ04UQpwfhj5z5kyRkJAg9Hq9GDFihCgtLW23jbNnz4rHHntMGAwGYTQaxeOPPy7q6uo8PnSRiIjk1NHPcUUIIfyYkVfFbrfDZDLBZrPx+hQRUQDq6Od4QIzuIyKi4MSQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFqdngWdiDpOCAE4nXBUVqJ21y7UHzmCNpsNqk6H0O7dYbzlFnRLT4eq10NR+Z2R6FIMKSIvaqmsRNX69Ti7bRuc9fXARfM51+3fjzObNiGiTx+YH34Y3QYMgEav92O1RPLhVzciL2k6cQJlf/oTqtauhbOurl1AublcaCgtRdmiRajevBnC6fR9oUQSY0gReZgQAg6rFSc//BD2b77p0Gtaa2pwetkynP3iCwiXy8sVEgUOhhSRhwmnE9YVK2DfuxfoROA4Gxpw6m9/Q31JiRerIwosDCkiD3NYrajeuvWqXttaXY1zX30Fl8Ph4aqIAhNDisjDKj74AOjCtaWz27ah6fhxzxVEFMAYUkQe1ma3d+n1rqYmOBsbzw9fJwpyDCkiCbXU1Pi7BCIpMKSIJNRy5szlh6wTBRmGFJGE6g8d6tTIQKJrFUOKSEL1JSX8vRQRGFJEHqcxGPxdAtE1gyFF5GHRd90FeGCyWI7uI2JIEXmcNiqq6xsRAm11dV3fDlGAY0gReZguNhZQlK5tRAi0VFV5piCiAMaQIvKwEJMJXYwoQAi0nj3riXKIAhpDisjDPHHzQuF0onbnTg9UQxTYGFJEkmqrrfV3CUR+x5AiIiJpMaSIPEwJCUHUL3/Z5e0IpxOulhYPVEQUuBhSRB6mqCrCevTo8nZcDgeHoVPQY0gReZqiIMQDv5VytbTAWV/vgYKIAhdDisjTFAVao7HLm3GcPo26Awc8UBBR4GJIEXmYoiiemRaprQ3O5mYPVEQUuBhSREQkLYYUkRcoGg0UrbbrGxKCE81SUGNIEXlBaGIiDKmpXd5OW10dRGurByoiCkwMKSIvUPV6aCIiurydNrudIUVBjSFF5AWqXu+Rmx+2nDnDwRMU1BhSRF6g6HTQhIV1eTv1Bw+i9dw5D1REFJgYUkReoChK1+8pRUQMKSLpcYQfBTGGFJGX6BMSoOr1Xd6Oq6nJA9UQBSaGFJGXhPfq5ZERfi3V1R6ohigwMaSIvCTEZPLID3odVVUeqIYoMDGkiLwkxGiE6oGQqjtwAOA1KQpSDCkiL9GEh0PRaLq8ncajRxlSFLQYUkReonhgJnSiYMd/RUQBQLS1+bsEIr9gSBF5kaLTdX0jLhdaa2u7vh2iAMSQIvKiuKysLm9DCMFh6BS0GFJEXqSLju7yNkRrK2x793qgGqLAw5Ai8iJtTEzX5/BzudBcVuaZgogCDEOKyItCTCZ/l0AU0BhSRF6keHAmdE4yS8GIIUUUAFwtLXA5HP4ug8jnGFJEXqSoKvTXXdfl7biam+Gsr/dARUSBhSFF5EWKVotu6eld3o6zuRltDQ0eqIgosIT4uwCia5miqtB2YRh6U1sbKpub4Tp8GLbPP0fcXXdBp9NBp9MhLCwMYWFh0Ol0Hr32RSQThhSRN6kqtFFRV/3yEpsNM7/5BmcdDoQWFMAYFYWoqCjExcUhKSkJPXv2RM+ePXHjjTfihhtuQFxcHEJC+M+arh38aybyJkXp0o0PNYoCFedH9jU5HGi0WmG1WlFSUgIAUFUVYWFhSEhIQFJSEgYMGIAHHngAQ4YMgcFggKIoPMqigNapa1K5ubkYMmQIunXrhvj4eDz00EMoLS1t16a5uRk5OTmIiYmBwWBAdnY2Kisr27UpKytDVlYWwsPDER8fj+nTp6ONE2jSNUhRlC7Nhp4QGopHe/XCxN698URmJsaPG4eMjAwMGjQIffr0QVxcHIQQOHr0KPLz8/HOO+/gwQcfxB133IFXX30VxcXFsNlsHL5OAatTR1L5+fnIycnBkCFD0NbWhhdffBEjR47E4cOHEfHjt8Xnn38eGzZswIoVK2AymTB16lT8+te/xvbt2wEATqcTWVlZMJvN2LFjB06fPo0JEyZAq9Vizpw5nn+HRAHMHB6OsTfcAABI+PWvkfgv/wKEhKC+vh4VFRU4duwYfvjhBxw8eBAHDhzAt99+i9raWhQXF6O4uBgffPABMjMzMWbMGNx5553uf6dEgUIRXfiKdebMGcTHxyM/Px933HEHbDYb4uLi8PHHH+M3v/kNAODIkSO46aabUFBQgOHDh2PTpk144IEHcOrUKSQkJAAAlixZgv/8z//EmTNnoOvArNF2ux0mkwk2mw1Go/FqyyfyiXMFBTg6d26Xb1wYPWIEkp96Cprw8HbLXS4XGhsbUVVVhWPHjmHz5s1Yt24dTpw4AYfDAVVVYTabce+992Ly5Mm45ZZboNVqeRqQ/Kqjn+NdGoJus9kAANE/jl4qLCxEa2srMjIy3G369euH5ORkFBQUAAAKCgrQv39/d0ABQGZmJux2Ow4dOnTZ/TgcDtjt9nYPokARnpICQ2pql7fjOHkSzubmnyxXVRUGgwEpKSm45557MGfOHHz11Vd4//33MWLECERHR+P06dP4y1/+gjFjxuCVV17B8ePH4XQ6u1wTkbdddUi5XC4899xzuO2225CWlgYAsFqt0Ol0iIyMbNc2ISEBVqvV3ebigLqw/sK6y8nNzYXJZHI/kpKSrrZsIp9Tw8K6NHjigobSUjh/5rdSFwZJhISEIDY2Fo888giWLVuG+fPnY/To0dDr9Thz5gzmzZuHJ598EmvWrIHD4eD1KpLaVYdUTk4ODh48iOXLl3uynsuaMWMGbDab+1FeXu71fRJ5iiY01CMh1VmKoiAmJga//e1vsWTJEsyfPx9paWlQFAX5+fmYOnUq5syZA7vdzqAiaV1VSE2dOhXr16/Htm3b0L17d/dys9mMlpYW1F5yF9HKykqYzWZ3m0tH+114fqHNpfR6PYxGY7sHUaBQdDqoer1nNuZydTpQVFVFfHw8nnjiCaxZswaTJ09GTEwMqqqq8Nprr2Hy5Mk4ePAgXC6XZ2ok8qBOhZQQAlOnTsXq1auxdetWpKSktFs/aNAgaLVa5OXluZeVlpairKwMFosFAGCxWFBcXIyqqip3my1btsBoNCLVA+ftiWTjyQEKrVd5PVZRFKiqipSUFMyZMwdz585F//790dbWhlWrVuH3v/89CgsLPVYnkad0KqRycnLw17/+FR9//DG6desG648/LGxqagIAmEwmTJo0CdOmTcO2bdtQWFiIxx9/HBaLBcOHDwcAjBw5EqmpqRg/fjz279+PzZs34+WXX0ZOTg70nvq2SXSNajlzpsvbCA8Px7hx47B06VLcc889AICvvvoKkydPxvbt29HW1sbTfySNToXU4sWLYbPZcNddd+G6665zPz755BN3m7feegsPPPAAsrOzcccdd8BsNmPVqlXu9RqNBuvXr4dGo4HFYsG4ceMwYcIEzJ4923PvikgyuthYKB6Yrqj+4MEub0NRFGi1WgwYMAAfffQRxo0bh4iICBQVFSEnJwd5eXk89UfS6NLvpPyFv5OiQFNXUoKjc+ag7cefbVytEJMJ6X/5i8dOIQohYLVa8eabb2LRokVwOBxIT0/Hu+++iyFDhnhkH0SX45PfSRFRx2hNJo8cSXmaoigwm82YNWsWpk6dCr1ej/379+PZZ5/lYAqSAkOKyAdCTCYoGo2/y7gsRVFgMBgwY8YMTJo0CeHh4dizZw9mzpyJ8vJyXp8iv2JIEfmAJizMMyElBFwtLV3fziUURYHJZMJ//Md/4P777wcAbN68GQsWLEBra6vH90fUUQwpIh/oykzoFxMuF1prajyyrUspioKkpCS89tprGDhwIBwOB959912sWLGCUyiR3zCkiAKIcDq9FlLA+aDq0aMHcnNzkZycjKamJsydOxf79+/naT/yC4YUka94YESey+GAbfduDxRzZYqi4Be/+AWefvppRERE4Ntvv8Wf//xn1NbWMqjI5xhSRD6SMGZM1zficqGlurrr2/kn9Ho9nnzySYwcORJOpxOffPIJ/vGPfzCkyOcYUkQ+orvC3JQyUhQFcXFxmD59OmJjY2Gz2TB//nycO3fO36VRkGFIEfmI9sf7rnWZED47ohk0aBAmTJgARVHwzTffYNWqVTyaIp9iSBH5SEi3bh7Zjqu5Ga4f58v0No1GgyeffBLp6elwOBx4//33cezYMZ/smwhgSBEFHGdT08/e/NCTLoz2y87ORmhoKEpKSvD555+jra3NJ/snYkgRBRhnQwPa6ut9tj+dTofRo0ejV69eqKurw4YNG1BdXc3TfuQTDCkiH1E0GoRERXV5O03Hj6Px++89UFHHKIqCtLQ03H///VBVFV9++SX279/vs/1TcGNIEfmIJiwMpsGDPbMxHx/FqKqK3/72tzAYDKivr8eaNWt4JEU+wZAi8hVVhdYDR1IX+DokUlNT3Tcv3bx5M86ePevT/VNwYkgR+Yii0XgspJzNzT4/mtLr9Xj44Yeh1WphtVqxdetWn+6fghNDioKeEAJ2ux1VVVXe3ZGiQBMW5pFNtdXWQvh40ldFUTBgwAB0794dra2tyM/P5yg/8jqGFAU9IQS2bNmC9evXe3W2b0/dTRcAHJWVED6+hYaiKOjevTvS09Phcrlw8OBBWK1Wn9ZAwYchRUGvpaUFf/3rX7FhwwbUeHGGcU8699VXPh2GfkF0dDQGDBgArVaL8vJyfPfddz6vgYKLfPezJvKxU6dOoaCgABqNBmVlZYiNjfXoUc/FdPHx0MXFoeXMGa9s39tCQkLQt29fGAwGVFdX48SJExBCeK2/iHgkRUFNCIFNmzahpqYGlZWV2LBhg1dHzWljYqCNjfXa9r1NURT069cPJpMJTU1N+P7779HihTsFE13AkKKgZrfbsWnTJjidTgghsGLFCtTV1Xltf5rwcI8NnhBtbX75rVKPHj0QHR0NIQR++OEHNDY2+rwGCh4MKQpaQgiUlJSgtLTUvez48ePYvn271z78NaGhHgupVj/dNsNgMOD6668HAFRUVMDhcPilDgoODCkKWm1tbfj6669x6tQp97Kmpib8/e9/99opLEWng6LTeWRbLd4eMn8FISEh6NmzJxRFQUVFBZqbm/1SBwUHhhQFrZqaGnz99dftPmSFENi7dy8OHTrklX12ZIBBRUMD1peXY9nRo/jHqVNouMJQc/u+fZ4ur0MURUFycjIAoLq6Gq0+HgpPwYWj+ygoCSFw8uRJ7N279yfrjh8/jt27d2PAgAHQaDQ+relYfT3+sG8fjtfXo9nphFGrRVpUFP44ZAi0avvvlPVeCtKOiI+Ph6IoaGho4A96yat4JEVBSQiBvLy8y/4YtampCRs3bsS5c+e8cm0qxGgE1J/+0ztaX4+ntm9Hic2GJqcTAoCttRXbq6rw7K5dOCvRabWYmBgAvp8/kIIPQ4qCksPhwKeffnrFD9kdO3bghx9+8Mq+Iy2Wyw6eePvQIdiucOpsd3U1tlx07czfIiIi/F0CBQmGFAWlQ4cOua876X4cyKAoCmJiYqCqKs6dO4f169d7Zd/ayEgoPjyN6A2+PA1KwY0hRUHH5XJh9erVaGlpwa233ornn38eGo0GiqJg1qxZGDt2LIxGI1auXOmVaZK0kZFAgH/Ie3OOQ6KLMaQo6FRWVmLnzp249957sWjRIowZMwbqj9eIYmJi8Mc//hEvvfQShBDYtm2bx6+7qGFhiBs16ifLs5KSoL3C6L+eBgPSo6M9WkdX8Ae85Csc3UdB59tvv8Xtt9+OSZMmISkpCaWlpVBVFa2trWhoaEBMTAxycnJw880348CBA2hqakJ4eLjH9q8oChIefBAOqxU127a5l2cmJgIAXt2/Hy1OJ1wANIqCSJ0ObwwZgh4GQ7vtJGRne6ymzrLZbH7bNwUXhhQFndTUVNx6663o1q0bFEVBREQEwsLC4HA4UFVVBSEEwsLCkJmZ6b4TraepYWGIslhg27sXzh+nYVIUBZmJiegeHo71J0/ibHMzehoMeCQlBTF6fbvX66+7DlEWi98mdq2srIQQAqGhoe6jUCJvYEhR0ImLi2v33GAwoFu3bqitrUVFRYV7uaqqiPLg7d4vpigKjLfeirj77kPl//2f+waGiqIgLSoKaT+z35CoKCSOH39+KLufXOin6OhoaLVav9VB1z5+BaKgFxYW5g6uo0eP+uy3P6pej4Rf/QrR99wDJaRj3xc1BgOue+QRRA4d6rcRgk6n032LDrPZDP0lR3lEnsSQoqCnqip69+4N4Pz1Kl9O86MJD0f3J55AQnY2dPHxV2ynaDQI69kTSU89hbj77oPqofn/rkZDQwPKysoAAMnJyQjz0IS5RJfD030U9FRVRd++fQGcn8/v1KlT6NWrl0/2rSgKQiIicN1vfgNjejrO7diB+kOH4LBa4XI4oDEYENq9O0yDB8M0eDDCkpP9foNBq9WKs2fPAjh/247Q0FC/1kPXNoYUBT1VVZGWlgatVouWlhbs27fPZyHlrkGvhyEtDRE33ghnU9P5e0W5XFA0GqhaLdTwcKgdPCXoTUIIfPvtt7DZbNDpdOjTpw9DiryKp/so6CmKgh49esBsNqO1tRW7du3yy5x0iqJA1euhjYyELjYW+vh46GJiEGI0ShFQwPkfQh8+fBh2ux2xsbHo1asXR/eRV/Gvi4KeoiiIj4/HDTfcAKfTieLiYtjtdn+XJaW6ujqUlJSgpaUFZrPZ50ecFHwYUkQ4f+uJAQMGQFVVHDt2DCUlJZzh+xJCCJw6dQpFRUUAgH79+rnvK0XkLQwpIgChoaEYPHgwjEYjTp48iaKiIrhcLn+XJRUhBEpKSvD9999Dq9Xirrvu4vBz8jqGFBHOn/IbOnQo4uPj0dTUhO3bt6Ouro5HUxdxOp3YsmULGhsbYTKZcPfdd/u7JAoCDCmiH6WkpOCOO+4AAOTl5bl/C0TnnTp1CuvWrQMA3HPPPTzVRz7BkCL6kUajwaOPPgqdTofKykqsXbvW3yVJQwiBVatW4cyZMwgNDUV2djbvKUU+wZAiusjAgQMxcOBAAMCyZctQXV3t54r8TwiB06dPY82aNXA6nejfv7+7j4i8jSFFdBGDwYAnnngCer0ex44dw0cffYS2tjZ/l+VXTqcT69atw4EDBxAaGooHH3wQ119/vd9nvqDgwJAiuohGo8Evf/lL3HLLLXA4HFi1ahV++OGHoB1AIYSA1WrFypUrYbfbkZKSglGjRkHnx7kDKbgwpIgukZKSgjFjxiA0NBRFRUX49NNP0dLS4u+y/EIIgU2bNmHHjh3QaDR44IEHcPPNN/u7LAoiDCmiS2i1WowfPx5paWloamrCkiVLcPjw4aA8mjpx4gRyc3PR3NyMlJQU/O53v+P9o8inGFJEl5GYmIgpU6ZAq9XixIkTmD9/PhobG/1dlk81NDTg7bffxsmTJ6HX6/Fv//ZvvBZFPseQIroMRVEwZswYZGZmQlEUrFu3DitWrAiaQRROpxMbN27EypUr4XK5cOeddyI7O5uTyZLP8S+O6AqioqIwffp09O3bF7W1tXjjjTewY8eOa366JCEEDh06hLlz58JqtSIpKQnPP/88kiW4lxUFH4YU0RUoioJBgwbhiSeeQHh4OI4cOYJ58+a5b/h3rbLb7Zg7dy72798PVVXxr//6r7jjjjsYUOQXDCminxEWFoYnn3wSWVlZAIDPP/8cs2fPRmNj4zU3kEIIgfr6erzxxhv49NNPoSgKMjMz8cwzz/DGhuQ3DCmif8JoNGLu3LmwWCxwuVxYunQp3njjjWtqAlohBBobG7Fw4UIsXLgQDocDgwcPxpw5cxAVFeXv8iiIMaSI/glFUZCUlISZM2eib9++aGxsxKJFi/CXv/wFbW1tAR9UQgi4XC4sW7YMb7/9Nmpra3H99dfjpZdeQmpqKk/zkV8xpIg6QKPR4O6778bcuXORmJiIqqoqzJo1C//7v/+LpqamgA0qIQSampqwdOlSvPzyy6iqqkJ0dDRyc3MxatQohEhy23oKXgwpog4KCQlBVlYW3njjDSQlJcFms+HFF1/Em2++ierq6oAMqrq6OsyfPx8vvfQSzpw5g8TERMyePRsPP/wwA4qkwL9Cok4aM2YMmpubMWvWLJSXl+PNN99ERUUFZs+ejdjYWOlPj10I03PnzmHu3Ll47733YLPZEB0djRdffBHjxo1jQJE0eCRF1AmKokCv1+Oxxx7DO++8gxtuuAF2ux3vv/8+xo0bh507d6KlpUXqoyqn04m9e/di0qRJeOedd2C329G9e3csWLAATz75JAwGg/RBS8GDIUXUSYqiQKvVIisrC3/6059w2223AQD+8Y9/YOLEiVi4cKGUv6USQqCmpgYffvghJk6ciLVr16K1tRVDhw7FokWL8PDDD0Or1TKgSCqKkPkr3xXY7XaYTCbYbDYYjUZ/l0NBzOl04vDhw3jttdewcuVKtLa2olu3bhg1ahR+//vfY/DgwX7/4L8wem///v1YsGABNmzYgJqaGmg0GowaNQr/9V//hZtvvpmn+MinOvo53qkjqcWLFyM9PR1GoxFGoxEWiwWbNm1yr29ubkZOTg5iYmJgMBiQnZ2NysrKdtsoKytDVlYWwsPDER8fj+nTpwfNfGh07dFoNEhLS8OSJUvw6quvolevXmhsbMTKlSvx0EMPYfr06SguLkZDQ4PPTwEKIdDc3IySkhLMnj0bWVlZ+Nvf/oZz586hR48emDlzJpYuXYr09HQGFEmrU0dS69atg0ajQZ8+fSCEwIcffoh58+Zh3759uPnmmzFlyhRs2LABS5cuhclkwtSpU6GqKrZv3w7g/LfOW265BWazGfPmzcPp06cxYcIEPPXUU5gzZ06Hi+aRFMmopaUFO3fuxJ///Gd8+umnaGhogKIouOGGG5CVlYUHHngAQ4cOhcFg8HotjY2NKC4uxrp167B27VocOXIETqcT4eHhyMjIwNSpU3H77bdDr9d7vRaiy+nw57jooqioKPHee++J2tpaodVqxYoVK9zrSkpKBABRUFAghBBi48aNQlVVYbVa3W0WL14sjEajcDgcHd6nzWYTAITNZutq+UQe5XK5RFVVlVi6dKkYMmSICA0NFaqqCq1WK3r06CGys7PF0qVLRUVFhXA4HMLpdAqXy9XlfTqdTuFwOITVahXLly8Xjz76qLjxxhuFTqcTqqoKnU4nBg4cKP70pz+JioqKLu+TqKs6+jl+1cf4TqcTK1asQENDAywWCwoLC9Ha2oqMjAx3m379+iE5ORkFBQUYPnw4CgoK0L9/fyQkJLjbZGZmYsqUKTh06BBuvfXWy+7L4XDA4XC0S2AiGSmKgtjYWIwfPx73338/VqxYgY8++gilpaUoLy/HyZMnsXbtWpjNZtx+++248847kZaWhvj4eJhMJhiNRuj1+p+9hiWEQGtrK+x2O86dO4fq6mocPnwYW7duRUFBASoqKuB0OiGEgNFoxE033YTf/OY3GD9+PGJiYqAoCgdHUMDodEgVFxfDYrGgubkZBoMBq1evRmpqKoqKiqDT6RAZGdmufUJCAqxWKwDAarW2C6gL6y+su5Lc3Fy88sornS2VyC8uhEBcXBwmT56M+++/H1988QU+++wzfPnll6iqqkJFRQU++eQT/P3vf0dsbCx69OiBpKQkdO/eHXFxcYiOjobRaERoaChCQkLgcrnQ3NyMuro6VFdXo7q6GhUVFThx4gTKyspQVVXlvuZ1Yd8WiwWjR4/G3XffjeTkZN4LigJSp0Oqb9++KCoqgs1mw8qVKzFx4kTk5+d7oza3GTNmYNq0ae7ndrsdSUlJXt0nkSeoqoqePXtiwoQJGD16NI4ePYqvvvoKa9euxcGDB9HY2IizZ8/izJkz2Lt3L4DzM1vodDrodDpoNBooigIhBJxOJ1pbW+FwONoNNlJVFVqtFhEREejduzcefPBB3HPPPejVqxdiYmIYThTQOh1SOp0OvXv3BgAMGjQIe/bswfz58/HII4+gpaUFtbW17Y6mKisrYTabAQBmsxm7d+9ut70Lo/8utLkcvV7PC7wU0FRVRXR0NKKjozFo0CDk5OTg6NGj2LNnD3bt2oWjR4+iuroa586dQ0NDAxobG9HY2OiewFZVVWg0Guh0OkRHRyMiIgImkwnx8fFITk7G0KFDMXz4cPTq1avd6UKe1qNA1+Vxpy6XCw6HA4MGDYJWq0VeXh6ys7MBAKWlpSgrK4PFYgEAWCwW/Pd//zeqqqoQHx8PANiyZQuMRiNSU1O7WgqR1C4ODr1ej5tuugk33XQTxo0bB5vNhjNnzqCyshLnzp1DXV0dmpqa0NraCpfLBVVVodPpEBoaCqPRiMjISMTFxbmvZWk0Gj+/OyLv6FRIzZgxA/fddx+Sk5NRV1eHjz/+GF988QU2b94Mk8mESZMmYdq0ae7z6c888wwsFguGDx8OABg5ciRSU1Mxfvx4vP7667BarXj55ZeRk5PDIyUKWqqqIioqClFRUbjxxhv9XQ6RVDoVUlVVVZgwYQJOnz4Nk8mE9PR0bN68Gffeey8A4K233oKqqsjOzobD4UBmZiYWLVrkfr1Go8H69esxZcoUWCwWREREYOLEiZg9e7Zn3xUREV0TOC0SERH5nFemRSIiIvIlhhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNLqUkjNnTsXiqLgueeecy9rbm5GTk4OYmJiYDAYkJ2djcrKynavKysrQ1ZWFsLDwxEfH4/p06ejra2tK6UQEdE16KpDas+ePXj33XeRnp7ebvnzzz+PdevWYcWKFcjPz8epU6fw61//2r3e6XQiKysLLS0t2LFjBz788EMsXboUs2bNuvp3QURE1yZxFerq6kSfPn3Eli1bxJ133imeffZZIYQQtbW1QqvVihUrVrjblpSUCACioKBACCHExo0bhaqqwmq1utssXrxYGI1G4XA4OrR/m80mAAibzXY15RMRkZ919HP8qo6kcnJykJWVhYyMjHbLCwsL0dra2m55v379kJycjIKCAgBAQUEB+vfvj4SEBHebzMxM2O12HDp06LL7czgcsNvt7R5ERHTtC+nsC5YvX45vvvkGe/bs+ck6q9UKnU6HyMjIdssTEhJgtVrdbS4OqAvrL6y7nNzcXLzyyiudLZWIiAJcp46kysvL8eyzz+Jvf/sbQkNDvVXTT8yYMQM2m839KC8v99m+iYjIfzoVUoWFhaiqqsLAgQMREhKCkJAQ5OfnY8GCBQgJCUFCQgJaWlpQW1vb7nWVlZUwm80AALPZ/JPRfheeX2hzKb1eD6PR2O5BRETXvk6F1IgRI1BcXIyioiL3Y/DgwRg7dqz7v7VaLfLy8tyvKS0tRVlZGSwWCwDAYrGguLgYVVVV7jZbtmyB0WhEamqqh94WERFdCzp1Tapbt25IS0trtywiIgIxMTHu5ZMmTcK0adMQHR0No9GIZ555BhaLBcOHDwcAjBw5EqmpqRg/fjxef/11WK1WvPzyy8jJyYFer/fQ2yIiomtBpwdO/DNvvfUWVFVFdnY2HA4HMjMzsWjRIvd6jUaD9evXY8qUKbBYLIiIiMDEiRMxe/ZsT5dCREQBThFCCH8X0Vl2ux0mkwk2m43Xp4iIAlBHP8c5dx8REUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSSvE3wVcDSEEAMBut/u5EiIiuhoXPr8vfJ5fSUCG1NmzZwEASUlJfq6EiIi6oq6uDiaT6YrrAzKkoqOjAQBlZWU/++aCnd1uR1JSEsrLy2E0Gv1djrTYTx3DfuoY9lPHCCFQV1eHxMTEn20XkCGlqucvpZlMJv4RdIDRaGQ/dQD7qWPYTx3DfvrnOnKQwYETREQkLYYUERFJKyBDSq/X4w9/+AP0er2/S5Ea+6lj2E8dw37qGPaTZynin43/IyIi8pOAPJIiIqLgwJAiIiJpMaSIiEhaDCkiIpJWQIbUwoUL0bNnT4SGhmLYsGHYvXu3v0vyqS+//BKjR49GYmIiFEXBmjVr2q0XQmDWrFm47rrrEBYWhoyMDHz33Xft2tTU1GDs2LEwGo2IjIzEpEmTUF9f78N34V25ubkYMmQIunXrhvj4eDz00EMoLS1t16a5uRk5OTmIiYmBwWBAdnY2Kisr27UpKytDVlYWwsPDER8fj+nTp6Otrc2Xb8WrFi9ejPT0dPcPTy0WCzZt2uRezz66vLlz50JRFDz33HPuZewrLxEBZvny5UKn04n3339fHDp0SDz11FMiMjJSVFZW+rs0n9m4caN46aWXxKpVqwQAsXr16nbr586dK0wmk1izZo3Yv3+/GDNmjEhJSRFNTU3uNqNGjRIDBgwQO3fuFF999ZXo3bu3eOyxx3z8TrwnMzNTfPDBB+LgwYOiqKhI3H///SI5OVnU19e720yePFkkJSWJvLw8sXfvXjF8+HDxi1/8wr2+ra1NpKWliYyMDLFv3z6xceNGERsbK2bMmOGPt+QVa9euFRs2bBDffvutKC0tFS+++KLQarXi4MGDQgj20eXs3r1b9OzZU6Snp4tnn33WvZx95R0BF1JDhw4VOTk57udOp1MkJiaK3NxcP1blP5eGlMvlEmazWcybN8+9rLa2Vuj1erFs2TIhhBCHDx8WAMSePXvcbTZt2iQURREVFRU+q92XqqqqBACRn58vhDjfJ1qtVqxYscLdpqSkRAAQBQUFQojzXwZUVRVWq9XdZvHixcJoNAqHw+HbN+BDUVFR4r333mMfXUZdXZ3o06eP2LJli7jzzjvdIcW+8p6AOt3X0tKCwsJCZGRkuJepqoqMjAwUFBT4sTJ5HDt2DFartV0fmUwmDBs2zN1HBQUFiIyMxODBg91tMjIyoKoqdu3a5fOafcFmswH4/5MTFxYWorW1tV0/9evXD8nJye36qX///khISHC3yczMhN1ux6FDh3xYvW84nU4sX74cDQ0NsFgs7KPLyMnJQVZWVrs+Afj35E0BNcFsdXU1nE5nu//JAJCQkIAjR474qSq5WK1WALhsH11YZ7VaER8f3259SEgIoqOj3W2uJS6XC8899xxuu+02pKWlATjfBzqdDpGRke3aXtpPl+vHC+uuFcXFxbBYLGhubobBYMDq1auRmpqKoqIi9tFFli9fjm+++QZ79uz5yTr+PXlPQIUU0dXIycnBwYMH8fXXX/u7FCn17dsXRUVFsNlsWLlyJSZOnIj8/Hx/lyWV8vJyPPvss9iyZQtCQ0P9XU5QCajTfbGxsdBoND8ZMVNZWQmz2eynquRyoR9+ro/MZjOqqqrarW9ra0NNTc01149Tp07F+vXrsW3bNnTv3t293Gw2o6WlBbW1te3aX9pPl+vHC+uuFTqdDr1798agQYOQm5uLAQMGYP78+eyjixQWFqKqqgoDBw5ESEgIQkJCkJ+fjwULFiAkJAQJCQnsKy8JqJDS6XQYNGgQ8vLy3MtcLhfy8vJgsVj8WJk8UlJSYDab2/WR3W7Hrl273H1ksVhQW1uLwsJCd5utW7fC5XJh2LBhPq/ZG4QQmDp1KlavXo2tW7ciJSWl3fpBgwZBq9W266fS0lKUlZW166fi4uJ2gb5lyxYYjUakpqb65o34gcvlgsPhYB9dZMSIESguLkZRUZH7MXjwYIwdO9b93+wrL/H3yI3OWr58udDr9WLp0qXi8OHD4umnnxaRkZHtRsxc6+rq6sS+ffvEvn37BADx5ptvin379okTJ04IIc4PQY+MjBSffvqpOHDggHjwwQcvOwT91ltvFbt27RJff/216NOnzzU1BH3KlCnCZDKJL774Qpw+fdr9aGxsdLeZPHmySE5OFlu3bhV79+4VFotFWCwW9/oLQ4ZHjhwpioqKxGeffSbi4uKuqSHDL7zwgsjPzxfHjh0TBw4cEC+88IJQFEV8/vnnQgj20c+5eHSfEOwrbwm4kBJCiP/5n/8RycnJQqfTiaFDh4qdO3f6uySf2rZtmwDwk8fEiROFEOeHoc+cOVMkJCQIvV4vRowYIUpLS9tt4+zZs+Kxxx4TBoNBGI1G8fjjj4u6ujo/vBvvuFz/ABAffPCBu01TU5P43e9+J6KiokR4eLj41a9+JU6fPt1uO8ePHxf33XefCAsLE7GxseLf//3fRWtrq4/fjfc88cQTokePHkKn04m4uDgxYsQId0AJwT76OZeGFPvKO3irDiIiklZAXZMiIqLgwpAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKS1v8D13+1+RjFh1wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "-120.60388566888659"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(play=True)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第7章-DQN算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
